// ----------------------------------------------------------------------
// Copyright (c) 2016, The Regents of the University of California All
// rights reserved.
// 
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
// 
//     * Redistributions of source code must retain the above copyright
//       notice, this list of conditions and the following disclaimer.
// 
//     * Redistributions in binary form must reproduce the above
//       copyright notice, this list of conditions and the following
//       disclaimer in the documentation and/or other materials provided
//       with the distribution.
// 
//     * Neither the name of The Regents of the University of California
//       nor the names of its contributors may be used to endorse or
//       promote products derived from this software without specific
//       prior written permission.
// 
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL REGENTS OF THE
// UNIVERSITY OF CALIFORNIA BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
// OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
// TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
// DAMAGE.
// ----------------------------------------------------------------------

#include "precomp.h"

// The trace message header (.tmh) file must be included in a source file
// before any WPP macro calls and after defining a WPP_CONTROL_GUIDS
// macro (defined in toaster.h). During the compilation, WPP scans the source
// files for DoTraceMessage() calls and builds a .tmh file which stores a unique
// data GUID for each message, the text resource string for each message,
// and the data types of the variables passed in for each message.  This file
// is automatically generated by the WPP preprocessor.
//
#include "riffa.tmh"


#ifdef ALLOC_PRAGMA
#pragma alloc_text (INIT, DriverEntry)
#pragma alloc_text (PAGE, RiffaEvtDeviceAdd)
#pragma alloc_text (PAGE, RiffaEvtDriverContextCleanup)
#pragma alloc_text (PAGE, RiffaEvtDevicePrepareHardware)
#pragma alloc_text (PAGE, RiffaEvtDeviceReleaseHardware)

#endif

/**
 * DriverEntry initializes the driver and is the first routine called by the
 * system after the driver is loaded. DriverEntry configures and creates a WDF driver
 * object.
 *	.
 * DriverObject - represents the instance of the function driver that is loaded
 * into memory. DriverObject is allocated by the system before the
 * driver is loaded, and it is released by the system after the system unloads
 * the function driver from memory.
 *
 * RegistryPath - represents the driver specific path in the Registry.
 * The function driver can use the path to store driver related data between
 * reboots. The path does not store hardware instance specific data.
 *
 * Returns STATUS_SUCCESS if successful, STATUS_UNSUCCESSFUL otherwise.
 */
NTSTATUS DriverEntry(IN PDRIVER_OBJECT  DriverObject, IN PUNICODE_STRING RegistryPath)
{
	NTSTATUS status = STATUS_SUCCESS;
	WDF_DRIVER_CONFIG config;
	WDF_OBJECT_ATTRIBUTES attributes;

	// Initialize WDF WPP tracing.
	WPP_INIT_TRACING(DriverObject, RegistryPath);

	// TraceEvents function is mapped to DoTraceMessage provided by
	// WPP by using a directive in the sources file.
	TraceEvents(TRACE_LEVEL_INFORMATION, DBG_INIT, "RIFFA Driver.");
	TraceEvents(TRACE_LEVEL_INFORMATION, DBG_INIT, "Built %s %s", __DATE__, __TIME__);

	// Initialize the Driver Config structure.
	WDF_DRIVER_CONFIG_INIT(&config, RiffaEvtDeviceAdd);

	// Register a cleanup callback so that we can call WPP_CLEANUP when
	// the framework driver object is deleted during driver unload.
	WDF_OBJECT_ATTRIBUTES_INIT(&attributes);
	attributes.EvtCleanupCallback = RiffaEvtDriverContextCleanup;

	// Create the WDFDRIVER object
	status = WdfDriverCreate(DriverObject, RegistryPath, &attributes, &config,
		WDF_NO_HANDLE);

	if (!NT_SUCCESS(status)) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: WdfDriverCreate failed\n");

		// Cleanup tracing here because DriverContextCleanup will not be called
		// as we have failed to create WDFDRIVER object itself.
		WPP_CLEANUP(DriverObject);
	}

	return status;
}



/******************************************************************************
 * PNP EVENT CALLBACKS
 *****************************************************************************/

/**
 * Called by the framework in response to AddDevice call from the PnP manager.
 * We create and initialize a WDF device object to represent a new instance of
 * our device.
 *	.
 * Driver - Handle to a framework driver object created in DriverEntry
 *
 * DeviceInit - Pointer to a framework-allocated WDFDEVICE_INIT structure.
 *
 * Returns STATUS_SUCCESS if successful, another status otherwise.
 */
NTSTATUS RiffaEvtDeviceAdd(IN WDFDRIVER Driver, IN PWDFDEVICE_INIT DeviceInit) {
	NTSTATUS status = STATUS_SUCCESS;
	WDF_PNPPOWER_EVENT_CALLBACKS pnpPowerCallbacks;
	WDF_OBJECT_ATTRIBUTES attributes;
	WDF_IO_QUEUE_CONFIG queueConfig;
	WDF_DMA_ENABLER_CONFIG dmaConfig;
	WDF_INTERRUPT_CONFIG intrConfig;
	WDFDEVICE  device;
	PDEVICE_EXTENSION devExt = NULL;

	UNREFERENCED_PARAMETER(Driver);

	PAGED_CODE();

	// Setup direct I/O for read & write operations
	WdfDeviceInitSetIoType(DeviceInit, WdfDeviceIoDirect);

	// Zero out the PnpPowerCallbacks structure.
	WDF_PNPPOWER_EVENT_CALLBACKS_INIT(&pnpPowerCallbacks);

	// Register the PnP Callbacks.
	pnpPowerCallbacks.EvtDevicePrepareHardware = RiffaEvtDevicePrepareHardware;
	pnpPowerCallbacks.EvtDeviceReleaseHardware = RiffaEvtDeviceReleaseHardware;
	WdfDeviceInitSetPnpPowerEventCallbacks(DeviceInit, &pnpPowerCallbacks);

	// Initialize FDO Request context.
	WDF_OBJECT_ATTRIBUTES_INIT_CONTEXT_TYPE(&attributes, REQUEST_EXTENSION);
	WdfDeviceInitSetRequestAttributes(DeviceInit, &attributes);

	// Initialize FDO Attributes.
	WDF_OBJECT_ATTRIBUTES_INIT_CONTEXT_TYPE(&attributes, DEVICE_EXTENSION);

	// Create the device
	status = WdfDeviceCreate(&DeviceInit, &attributes, &device);
	if (!NT_SUCCESS(status)) {
		// Device Initialization failed.
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: DeviceCreate failed\n");
		return status;
	}

	// Get the DeviceExtension and initialize it. RiffaGetDeviceContext is an
	// inline function defined by the WDF_DECLARE_CONTEXT_TYPE_WITH_NAME macro
	// in the private header file. This function will do the type checking and
	// return the device context. If you pass a wrong object a wrong object handle
	// it will return NULL and assert if run under framework verifier mode.
	devExt = RiffaGetDeviceContext(device);
	memset(devExt->IntrData, 0, 2 * RIFFA_MAX_NUM_CHNLS * sizeof(INTR_CHNL_DIR_DATA));
	memset(devExt->Chnl, 0, 2 * RIFFA_MAX_NUM_CHNLS * sizeof(CHNL_DIR_STATE));
	devExt->Device = device;

	// Create a new IO Queue for IRP_MJ_DEVICE_CONTROL requests.
	WDF_IO_QUEUE_CONFIG_INIT(&queueConfig, WdfIoQueueDispatchParallel);
	queueConfig.EvtIoDeviceControl = RiffaEvtIoDeviceControl;
	status = WdfIoQueueCreate(device, &queueConfig, WDF_NO_OBJECT_ATTRIBUTES,
		&devExt->IoctlQueue);
	if(!NT_SUCCESS(status)) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: WdfIoQueueCreate failed\n");
		return status;
	}

	// Set the Ioctl Queue forwarding for IRP_MJ_DEVICE_CONTROL requests.
	status = WdfDeviceConfigureRequestDispatching(device, devExt->IoctlQueue,
		WdfRequestTypeDeviceControl);
	if(!NT_SUCCESS(status)) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: DeviceConfigureRequestDispatching failed\n");
		return status;
	}

	// Create a WDFINTERRUPT object.
	WDF_INTERRUPT_CONFIG_INIT(&intrConfig, RiffaEvtInterruptIsr, RiffaEvtInterruptDpc);
	intrConfig.AutomaticSerialization = TRUE;
	status = WdfInterruptCreate(device, &intrConfig, WDF_NO_OBJECT_ATTRIBUTES,
		&devExt->Interrupt);
	if(!NT_SUCCESS(status)) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: WdfInterruptCreate failed\n");
		return status;
	}

	// Create a new DMA Enabler instance. Use Scatter/Gather, 64-bit Addresses,
	// Duplex-type profile, 4-byte alignment.
	WdfDeviceSetAlignmentRequirement(device, FILE_LONG_ALIGNMENT);
	WDF_DMA_ENABLER_CONFIG_INIT(&dmaConfig, WdfDmaProfileScatterGather64Duplex,
		RIFFA_MAX_TNFR_LEN);
	status = WdfDmaEnablerCreate(device, &dmaConfig, WDF_NO_OBJECT_ATTRIBUTES,
		&devExt->DmaEnabler);
	if (!NT_SUCCESS (status)) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: WdfDmaEnablerCreate failed\n");
		return status;
	}

	// Tell the Framework that this device will need an interface
	// NOTE: See the note in public.h concerning this GUID value.
	status = WdfDeviceCreateDeviceInterface(device, &GUID_RIFFA_INTERFACE, NULL);
	if (!NT_SUCCESS(status)) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: DeviceCreateDeviceInterface failed\n");
		return status;
	}

	return status;
}



/**
 * Free all the resources allocated in DriverEntry.
 *	.
 * Driver - handle to a WDF Driver object.
 */
VOID RiffaEvtDriverContextCleanup(IN WDFDRIVER Driver) {
	PAGED_CODE ();

	WPP_CLEANUP(WdfDriverWdmGetDriverObject(Driver));
}



/**
 * Performs whatever initialization is needed to setup the device, setting up
 * a DMA channel or mapping any I/O port resources.  This will only be called
 * as a device starts or restarts, not every time the device moves into the D0
 * state.
 *	.
 * Device - A handle to the WDFDEVICE
 *
 * Resources - The raw PnP resources associated with the device.
 *
 * ResourcesTranslated - The translated PnP resources associated with the device.
 *
 * Returns STATUS_SUCCESS if successful, another status otherwise. Failure will
 * result in the device stack being torn down.
 */
NTSTATUS RiffaEvtDevicePrepareHardware(WDFDEVICE Device, WDFCMRESLIST Resources,
	WDFCMRESLIST ResourcesTranslated) {
	NTSTATUS status;
	PDEVICE_EXTENSION devExt;
	PTIMER_EXTENSION timerExt;
	WDF_TIMER_CONFIG timerConfig;
	WDF_IO_QUEUE_CONFIG queueConfig;
	WDF_OBJECT_ATTRIBUTES attributes;
	PCM_PARTIAL_RESOURCE_DESCRIPTOR desc;
	BOOLEAN foundBar0 = FALSE;
	UINT32 info;
	UINT32 i;

	UNREFERENCED_PARAMETER(Resources);

	PAGED_CODE();

	// Get the device extension
	devExt = RiffaGetDeviceContext(Device);

	// Parse the resource list and save the resource information.
	for (i=0; i < WdfCmResourceListGetCount(ResourcesTranslated); i++) {
		desc = WdfCmResourceListGetDescriptor(ResourcesTranslated, i);
		if(!desc) {
			DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
				"riffa: WdfResourceCmGetDescriptor failed\n");
			return STATUS_DEVICE_CONFIGURATION_ERROR;
		}

		switch (desc->Type) {

			case CmResourceTypeMemory:
				if (!foundBar0 && desc->u.Memory.Length == 0x400) {
					// Map in the Registers Memory resource: BAR0
					devExt->Bar0Length = desc->u.Memory.Length;
					devExt->Bar0 = (PULONG)MmMapIoSpace(desc->u.Memory.Start,
						desc->u.Memory.Length, MmNonCached);
					if (!devExt->Bar0) {
						DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
							"riffa: unable to map BAR memory %08I64X, length %d",
							desc->u.Memory.Start.QuadPart, desc->u.Memory.Length);
						return STATUS_INSUFFICIENT_RESOURCES;
					}
					foundBar0 = TRUE;
				}
				break;

			default:
				break;
		}
	}

	// Make sure we found BAR0
	if (!foundBar0) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL, "riffa: missing BAR0\n");
		return STATUS_DEVICE_CONFIGURATION_ERROR;
	}

	DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_TRACE_LEVEL,
		"riffa: BAR0 address: %p\n", devExt->Bar0);
	DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_TRACE_LEVEL,
		"riffa: BAR0 length: %u\n", devExt->Bar0Length);

	// Read the device ids
	status = RiffaReadHardwareIds(devExt);
	if(!NT_SUCCESS(status))
		return status;

	// Read the configuration register (also resets the device)
	info = READ_REGISTER_ULONG(devExt->Bar0 + RIFFA_INFO_REG);
	devExt->NumChnls = (info & 0xF);
	devExt->MaxNumScatterGatherElems = RIFFA_MIN_NUM_SG_ELEMS*((info>>19) & 0xF);

	DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_TRACE_LEVEL,
		"riffa: found FPGA with name: %s\n", devExt->Name);
	DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_TRACE_LEVEL,
		"riffa: vendor id: 0x%04X\n", devExt->VendorId);
	DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_TRACE_LEVEL,
		"riffa: device id: 0x%04X\n", devExt->DeviceId);
	DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_TRACE_LEVEL,
		"riffa: number of channels: %d\n", (info & 0xF));
	DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_TRACE_LEVEL,
		"riffa: bus interface width: %d\n", ((info>>19) & 0xF)<<5);
	DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_TRACE_LEVEL,
		"riffa: bus master enabled: %d\n", ((info>>4) & 0x1));
	DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_TRACE_LEVEL,
		"riffa: negotiated link width: %d\n", ((info>>5) & 0x3F));
	DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_TRACE_LEVEL,
		"riffa: negotiated link rate: %d MTs\n", ((info>>11) & 0x3)*2500);
	DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_TRACE_LEVEL,
		"riffa: max downstream payload: %d bytes\n", 128<<((info>>13) & 0x7));
	DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_TRACE_LEVEL,
		"riffa: max upstream payload: %d bytes\n", 128<<((info>>16) & 0x7));

	// Check for bus master enabled
	if (((info>>4) & 0x1) != 1) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: bus master not enabled!\n");
		return STATUS_DEVICE_CONFIGURATION_ERROR;
	}

	// Check for valid link parameters
	if (((info>>5) & 0x3F) == 0 || ((info>>11) & 0x3) == 0) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: bad link parameters!\n");
		return STATUS_DEVICE_CONFIGURATION_ERROR;
	}

	// Check for valid number of channels
	if ((info & 0xF) == 0 || (info & 0xF) > RIFFA_MAX_NUM_CHNLS) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: bad number of channels!\n");
		return STATUS_DEVICE_CONFIGURATION_ERROR;
	}

	// Check for valid bus width
	if (((info>>19) & 0xF) == 0 || ((info>>19) & 0xF) > RIFFA_MAX_BUS_WIDTH_PARAM) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: bad bus width!\n");
		return STATUS_DEVICE_CONFIGURATION_ERROR;
	}

    // Allocate IO queues, common buffers, DMA transactions, spin locks, timers for each channel.
	for (i = 0; i < devExt->NumChnls; i++) {
		// Create a new manual IO Queue for pending requests.
		WDF_IO_QUEUE_CONFIG_INIT(&queueConfig, WdfIoQueueDispatchManual);
		status = WdfIoQueueCreate(Device, &queueConfig, WDF_NO_OBJECT_ATTRIBUTES,
			&devExt->Chnl[i].PendingQueue);
		if(!NT_SUCCESS(status)) {
			DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
				"riffa: WdfIoQueueCreate failed\n");
			return status;
		}

		WDF_IO_QUEUE_CONFIG_INIT(&queueConfig, WdfIoQueueDispatchManual);
		status = WdfIoQueueCreate(Device, &queueConfig, WDF_NO_OBJECT_ATTRIBUTES,
			&devExt->Chnl[RIFFA_MAX_NUM_CHNLS + i].PendingQueue);
		if(!NT_SUCCESS(status)) {
			DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
				"riffa: WdfIoQueueCreate failed\n");
			return status;
		}

		// Common buffer creation, not cached.
		status = WdfCommonBufferCreate(devExt->DmaEnabler, RIFFA_MIN_SG_BUF_SIZE*((info>>19) & 0xF),
			WDF_NO_OBJECT_ATTRIBUTES, &devExt->Chnl[i].CommonBuffer);
		if (!NT_SUCCESS(status)) {
			DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
				"riffa: WdfCommonBufferCreate failed\n");
			return status;
		}
		devExt->Chnl[i].CommonBufferBase =
			WdfCommonBufferGetAlignedVirtualAddress(devExt->Chnl[i].CommonBuffer);
		devExt->Chnl[i].CommonBufferBaseLA =
			WdfCommonBufferGetAlignedLogicalAddress(devExt->Chnl[i].CommonBuffer);

		status = WdfCommonBufferCreate(devExt->DmaEnabler, RIFFA_MIN_SG_BUF_SIZE*((info>>19) & 0x1F),
			WDF_NO_OBJECT_ATTRIBUTES, &devExt->Chnl[RIFFA_MAX_NUM_CHNLS + i].CommonBuffer);
		if (!NT_SUCCESS(status)) {
			DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
				"riffa: WdfCommonBufferCreate failed\n");
			return status;
		}
		devExt->Chnl[RIFFA_MAX_NUM_CHNLS + i].CommonBufferBase =
			WdfCommonBufferGetAlignedVirtualAddress(devExt->Chnl[RIFFA_MAX_NUM_CHNLS + i].CommonBuffer);
		devExt->Chnl[RIFFA_MAX_NUM_CHNLS + i].CommonBufferBaseLA =
			WdfCommonBufferGetAlignedLogicalAddress(devExt->Chnl[RIFFA_MAX_NUM_CHNLS + i].CommonBuffer);

		// Since we are using sequential queue and processing one request at
		// a time, we will create transaction objects upfront and reuse them
		// to do DMA transfer. Transactions objects are parented to the DMA
		// enabler object by default. They will be deleted along with the DMA
		// enabler object. No need to delete them explicitly.
		WDF_OBJECT_ATTRIBUTES_INIT(&attributes);
		status = WdfDmaTransactionCreate(devExt->DmaEnabler, &attributes,
			&devExt->Chnl[i].DmaTransaction);
		if(!NT_SUCCESS(status)) {
			DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
				"riffa: WdfDmaTransactionCreate failed\n");
			return status;
		}

		WDF_OBJECT_ATTRIBUTES_INIT(&attributes);
		status = WdfDmaTransactionCreate(devExt->DmaEnabler, &attributes,
			&devExt->Chnl[RIFFA_MAX_NUM_CHNLS + i].DmaTransaction);
		if(!NT_SUCCESS(status)) {
			DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
				"riffa: WdfDmaTransactionCreate failed\n");
			return status;
		}

		// Create spinlocks for each direction. Make the parent the IOCTL queue
		// so that the spinlocks will be automatically deleted when the queue is.
		WDF_OBJECT_ATTRIBUTES_INIT(&attributes);
		attributes.ParentObject = devExt->IoctlQueue;
		status = WdfSpinLockCreate(&attributes,
			&devExt->Chnl[i].SpinLock);
		if (!NT_SUCCESS(status)) {
			DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
				"riffa: WdfSpinLockCreate failed\n");
			return status;
		}

		WDF_OBJECT_ATTRIBUTES_INIT(&attributes);
		attributes.ParentObject = devExt->IoctlQueue;
		status = WdfSpinLockCreate(&attributes,
			&devExt->Chnl[RIFFA_MAX_NUM_CHNLS + i].SpinLock);
		if (!NT_SUCCESS(status)) {
			DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
				"riffa: WdfSpinLockCreate failed\n");
			return status;
		}

		// Create a timer for each direction. Set the parent as the IOCTL queue
		// so that we can get the device extension and have the timer deleted
		// when the queue is. Also set the timer's context to have the chnl num.
		WDF_TIMER_CONFIG_INIT(&timerConfig, RiffaEvtTimerFunc);
		WDF_OBJECT_ATTRIBUTES_INIT_CONTEXT_TYPE(&attributes, TIMER_EXTENSION);
		attributes.ParentObject = devExt->IoctlQueue;
		status = WdfTimerCreate(&timerConfig, &attributes,
			&devExt->Chnl[i].Timer);
		if (!NT_SUCCESS(status)) {
			DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
				"riffa: WdfTimerCreate failed\n");
			return status;
		}
		timerExt = RiffaGetTimerContext(devExt->Chnl[i].Timer);
		timerExt->Chnl = i;

		WDF_TIMER_CONFIG_INIT(&timerConfig, RiffaEvtTimerFunc);
		WDF_OBJECT_ATTRIBUTES_INIT_CONTEXT_TYPE(&attributes, TIMER_EXTENSION);
		attributes.ParentObject = devExt->IoctlQueue;
		status = WdfTimerCreate(&timerConfig, &attributes,
			&devExt->Chnl[RIFFA_MAX_NUM_CHNLS + i].Timer);
		if (!NT_SUCCESS(status)) {
			DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
				"riffa: WdfTimerCreate failed\n");
			return status;
		}
		timerExt = RiffaGetTimerContext(devExt->Chnl[RIFFA_MAX_NUM_CHNLS + i].Timer);
		timerExt->Chnl = RIFFA_MAX_NUM_CHNLS + i;
	}

	// Create spill buffer, not cached.
	status = WdfCommonBufferCreate(devExt->DmaEnabler, RIFFA_SPILL_BUF_SIZE,
		WDF_NO_OBJECT_ATTRIBUTES, &devExt->SpillBuffer);
	if (!NT_SUCCESS(status)) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: WdfCommonBufferCreate failed\n");
		return status;
	}
	devExt->SpillBufferBase =
		WdfCommonBufferGetAlignedVirtualAddress(devExt->SpillBuffer);
	devExt->SpillBufferBaseLA =
		WdfCommonBufferGetAlignedLogicalAddress(devExt->SpillBuffer);

	return STATUS_SUCCESS;
}



/**
 * Unmap the resources that were mapped in RiffaEvtDevicePrepareHardware.
 * This will only be called when the device stopped for resource rebalance,
 * surprise-removed or query-removed.
 *	.
 * Device - A handle to the WDFDEVICE
 *
 * ResourcesTranslated - The translated PnP resources associated with the device.
 *
 * Returns STATUS_SUCCESS if successful, another status otherwise. Failure will
 * result in the device stack being torn down.
 */
NTSTATUS RiffaEvtDeviceReleaseHardware(IN WDFDEVICE Device, IN WDFCMRESLIST ResourcesTranslated) {
	PDEVICE_EXTENSION devExt;
	NTSTATUS status = STATUS_SUCCESS;

	UNREFERENCED_PARAMETER(ResourcesTranslated);

	PAGED_CODE();

	// Get the device extension
	devExt = RiffaGetDeviceContext(Device);

	// Unmap the I/O address space BAR0
	if (devExt->Bar0) {
		MmUnmapIoSpace(devExt->Bar0, devExt->Bar0Length);
		devExt->Bar0 = NULL;
	}

	return status;
}



/**
 * Attempts to read the hardware ids (vendor id, device id, bus num,
 * device num, function num) and set it in the PDEVICE_EXTENSION object.
 *	.
 * DevExt - Pointer to the Device Extension
 *
 * Returns STATUS_SUCCESS if successful, another status otherwise.
 */
NTSTATUS RiffaReadHardwareIds(IN PDEVICE_EXTENSION DevExt) {
	NTSTATUS status;
	PDEVICE_OBJECT pdo;
	UINT32 length;
	UINT32 pciBus;
	UINT32 pciAddr;
	USHORT pciDev;
	USHORT pciFxn;
	WCHAR hwIds[2048];
	wchar_t * id;

	// Initialize the values
	DevExt->DeviceId = 0;
	DevExt->VendorId = 0;
	DevExt->Name[0] = 0;

	// Get the PDO from our FDO.
	pdo = WdfDeviceWdmGetPhysicalDevice(DevExt->Device);

	// Read DevicePropertyBusNumber.
	status = IoGetDeviceProperty(pdo, DevicePropertyBusNumber, sizeof(UINT32),
		(PVOID)&pciBus, &length);
	if(!NT_SUCCESS(status)) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: IoGetDeviceProperty failed\n");
		return status;
	}

	// Read DevicePropertyAddress (contains device and function).
	status = IoGetDeviceProperty(pdo, DevicePropertyAddress, sizeof(UINT32),
		(PVOID)&pciAddr, &length);
	if(!NT_SUCCESS(status)) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: IoGetDeviceProperty failed\n");
		return status;
	}

	// Parse the function and device values and create a name.
	pciFxn = (USHORT)((pciAddr) & 0x0000FFFF);
	pciDev = (USHORT)(((pciAddr)>>16) & 0x0000FFFF);
	sprintf_s(DevExt->Name, 16, "%02x:%02x:%01x", pciBus, pciDev, pciFxn);

	// Read Device HardwareID. Should be something like:
	// PCI\VEN_10B5&DEV_515A&SUBSYS_905610B5&REV_02
	status = IoGetDeviceProperty(pdo, DevicePropertyHardwareID, 2048,
		(PVOID)hwIds, &length);
	if(!NT_SUCCESS(status)) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: IoGetDeviceProperty failed\n");
		return status;
	}

	// Null terminate the string.
	hwIds[length] = 0; // Add null-char

	// Search for the device id.
	id = wcsstr(hwIds, L"DEV_");
	if (id == NULL) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: bad DevicePropertyHardwareID, no device id in: %S\n", hwIds);
		return STATUS_DEVICE_CONFIGURATION_ERROR;
	}

	// Null terminate after the 4 char id and convert into an int.
	hwIds[((int)(id - hwIds)) + 8] = 0; // Add null-char
	swscanf_s(id + 4, L"%x", &DevExt->DeviceId);

	// Search for the vendor id.
	id = wcsstr(hwIds, L"VEN_");
	if (id == NULL) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: bad DevicePropertyHardwareID, no vendor id in: %S\n", hwIds);
		return STATUS_DEVICE_CONFIGURATION_ERROR;
	}
	hwIds[((int)(id - hwIds)) + 8] = 0; // Add null-char
	swscanf_s(id + 4, L"%x", &DevExt->VendorId);

	return STATUS_SUCCESS;
}



/******************************************************************************
 * INTERRUPT FUNCTIONS
 *****************************************************************************/

/**
 * Interrupt handler for this driver. Called at DIRQL level when the
 * device or another device sharing the same interrupt line asserts
 * the interrupt. The driver first checks the device to make sure whether
 * this interrupt is generated by its device and if so clear the interrupt
 * register to disable further generation of interrupts and queue a
 * DPC to do other I/O work related to interrupt - such as reading
 * the device memory, starting a DMA transaction, coping it to
 * the request buffer and completing the request, etc.
 *	.
 * Interupt - Handle to WDFINTERRUPT Object for this device.
 *
 * MessageID - MSI message ID (always 0 in this configuration)
 *
 * Returns TRUE if this device generated the interrupt, FALSE otherwise.
 */
BOOLEAN RiffaEvtInterruptIsr(IN WDFINTERRUPT Interrupt, IN ULONG MessageID) {
	PDEVICE_EXTENSION devExt;
	UINT32 vect0 = 0;
	UINT32 vect1 = 0;
	BOOLEAN recog = FALSE;

	UNREFERENCED_PARAMETER(MessageID);

	devExt = RiffaGetDeviceContext(WdfInterruptGetDevice(Interrupt));

	// Read the interrupt register
	vect0 = READ_REGISTER_ULONG(devExt->Bar0 + RIFFA_IRQ_0_REG);
	if (devExt->NumChnls > 6)
		vect1 = READ_REGISTER_ULONG(devExt->Bar0 + RIFFA_IRQ_1_REG);

	// Process the interrupt vector(s)
	recog = RiffaProcessInterrupt(devExt, 0, vect0);
	if (devExt->NumChnls > 6)
		recog = (recog | RiffaProcessInterrupt(devExt, 6, vect1));

	// Queue a callback for the DPC if needed.
	if (recog)
		WdfInterruptQueueDpcForIsr(Interrupt);

	return recog;
}



/**
 * Processes the interrupt vector read from the device during the interrupt
 * service routine. The vectors are organized from right to left (LSB to MSB) as:
 * [ 0] TX_TXN			 for channel 0 in VECT_0, channel 6 in VECT_1
 * [ 1] TX_SG_BUF_RECVD	 for channel 0 in VECT_0, channel 6 in VECT_1
 * [ 2] TX_TXN_DONE		 for channel 0 in VECT_0, channel 6 in VECT_1
 * [ 3] RX_SG_BUF_RECVD	 for channel 0 in VECT_0, channel 6 in VECT_1
 * [ 4] RX_TXN_DONE		 for channel 0 in VECT_0, channel 6 in VECT_1
 * ...
 * [25] TX_TXN			 for channel 5 in VECT_0, channel 11 in VECT_1
 * [26] TX_SG_BUF_RECVD	 for channel 5 in VECT_0, channel 11 in VECT_1
 * [27] TX_TXN_DONE		 for channel 5 in VECT_0, channel 11 in VECT_1
 * [28] RX_SG_BUF_RECVD	 for channel 5 in VECT_0, channel 11 in VECT_1
 * [29] RX_TXN_DONE		 for channel 5 in VECT_0, channel 11 in VECT_1
 * Positions 30 - 31 in both VECT_0 and VECT_1 are zero.
 *	.
 * DevExt - Pointer to the Device Extension
 *
 * Offset - Channel offset
 *
 * Vect - Interrupt vector
 *
 * Returns TRUE if a DPC should be queued to do more work, FALSE otherwise.
 */
BOOLEAN RiffaProcessInterrupt(IN PDEVICE_EXTENSION DevExt, IN UINT32 Offset,
	IN UINT32 Vect) {
	BOOLEAN queueDpc = FALSE;
	UINT32 offlast;
	UINT32 len;
	int i;
	int chnl;

	if (Vect & 0xC0000000) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: fpga:%s, received invalid interrupt:0x%08x\n", DevExt->Name, Vect);
		return FALSE;
	}

	for (i = 0; i < 6 && (i+Offset) < DevExt->NumChnls; i++) {
		chnl = i + Offset;

		// New TX (PC receive) transaction.
		if (Vect & (1<<((5*i)+0))) {
			queueDpc = TRUE;
			// Read the offset/last and length
			offlast = READ_REGISTER_ULONG(DevExt->Bar0 + CHNL_REG(chnl, RIFFA_TX_OFFLAST_REG));
			len = READ_REGISTER_ULONG(DevExt->Bar0 + CHNL_REG(chnl, RIFFA_TX_LEN_REG));
			DevExt->IntrData[RIFFA_MAX_NUM_CHNLS + chnl].OffLast = offlast;
			DevExt->IntrData[RIFFA_MAX_NUM_CHNLS + chnl].Length = len;
			DevExt->IntrData[RIFFA_MAX_NUM_CHNLS + chnl].NewTxn = TRUE;
		}

		// TX (PC receive) scatter gather buffer is read.
		if (Vect & (1<<((5*i)+1))) {
			queueDpc = TRUE;
			DevExt->IntrData[RIFFA_MAX_NUM_CHNLS + chnl].SgRead = TRUE;
		}

		// TX (PC receive) transaction done.
		if (Vect & (1<<((5*i)+2))) {
			queueDpc = TRUE;
			DevExt->IntrData[RIFFA_MAX_NUM_CHNLS + chnl].Done = TRUE;
		}

		// RX (PC send) scatter gather buffer is read.
		if (Vect & (1<<((5*i)+3))) {
			queueDpc = TRUE;
			DevExt->IntrData[chnl].SgRead = TRUE;
		}

		// RX (PC send) transaction done.
		if (Vect & (1<<((5*i)+4))) {
			queueDpc = TRUE;
			DevExt->IntrData[chnl].Done = TRUE;
		}
	}

	return queueDpc;
}



/**
 * DPC callback for ISR. Please note that on a multiprocessor system,
 * you could have more than one DPCs running simulataneously on
 * multiple processors. So if you are accesing any global resources
 * make sure to synchrnonize the accesses with a spinlock.
 *	.
 * Interupt - Handle to WDFINTERRUPT Object for this device.
 *
 * Device - WDFDEVICE object passed to InterruptCreate
 */
VOID RiffaEvtInterruptDpc(IN WDFINTERRUPT Interrupt, IN WDFDEVICE Device) {
	NTSTATUS status;
	BOOLEAN txnComplete;
	BOOLEAN cont;
	PDEVICE_EXTENSION devExt;
	INTR_CHNL_DIR_DATA intrData[2 * RIFFA_MAX_NUM_CHNLS];
	UINT32 chnl;
	UINT32 tnfr;
	UINT64 length;
	LONG doneReqd;
	WDFREQUEST request;

	UNREFERENCED_PARAMETER(Device);

	devExt = RiffaGetDeviceContext(WdfInterruptGetDevice(Interrupt));

	// Acquire this device's InterruptSpinLock.
	WdfInterruptAcquireLock(Interrupt);

	// Copy over the values and zero them out.
	memcpy(&intrData, devExt->IntrData, 2 * RIFFA_MAX_NUM_CHNLS * sizeof(INTR_CHNL_DIR_DATA));
	memset(devExt->IntrData, 0, 2 * RIFFA_MAX_NUM_CHNLS * sizeof(INTR_CHNL_DIR_DATA));

	// Release our interrupt spinlock
	WdfInterruptReleaseLock(Interrupt);

	// Update state for each channel
	for (chnl = 0; chnl < devExt->NumChnls; chnl++) {
		// Send more scatter gather elements.
		if (intrData[RIFFA_MAX_NUM_CHNLS + chnl].SgRead == TRUE) {
			KdPrintEx((DPFLTR_IHVDRIVER_ID, DPFLTR_TRACE_LEVEL,
				"riffa: fpga:%s chnl:%d, recv sg buf read\n", devExt->Name, chnl));
			RiffaProgramScatterGather(devExt, RIFFA_MAX_NUM_CHNLS + chnl);
		}

		// Finished with upstream transfer.
		if (intrData[RIFFA_MAX_NUM_CHNLS + chnl].Done == TRUE) {
			// Check if we've requested a done (in the event of a split transaction)
			doneReqd = InterlockedExchange(&devExt->Chnl[RIFFA_MAX_NUM_CHNLS + chnl].ReqdDone, 0);
			// Check if this DMA operation has completed. We might need to start
			// another transfer if there is still data to be transfered in the request.
			txnComplete = WdfDmaTransactionDmaCompleted(
				devExt->Chnl[RIFFA_MAX_NUM_CHNLS + chnl].DmaTransaction, &status);
			if (txnComplete) {
				KdPrintEx((DPFLTR_IHVDRIVER_ID, DPFLTR_TRACE_LEVEL,
					"riffa: fpga:%s chnl:%d, recv txn done\n", devExt->Name, chnl));

				// Read the actual transfer length
				tnfr = READ_REGISTER_ULONG(devExt->Bar0 + CHNL_REG(chnl, RIFFA_TX_TNFR_LEN_REG));
				RiffaTransactionComplete(devExt, RIFFA_MAX_NUM_CHNLS + chnl, tnfr, status);
			}
			else {
				if (doneReqd == 0) {
					// Not complete and not expecting a done signal. Must be an error.
					// End the transaction early.
					KdPrintEx((DPFLTR_IHVDRIVER_ID, DPFLTR_TRACE_LEVEL,
						"riffa: fpga:%s chnl:%d, recv txn done\n", devExt->Name, chnl));

					// Read the actual transfer length
					tnfr = READ_REGISTER_ULONG(devExt->Bar0 + CHNL_REG(chnl, RIFFA_TX_TNFR_LEN_REG));
					RiffaTransactionComplete(devExt, RIFFA_MAX_NUM_CHNLS + chnl, tnfr,
						STATUS_TRANSACTION_ABORTED);
				}
				else {
					KdPrintEx((DPFLTR_IHVDRIVER_ID, DPFLTR_TRACE_LEVEL,
						"riffa: fpga:%s chnl:%d, recv txn split, registers remapped\n", devExt->Name, chnl));
				}
			}
		}

		// New upstream transfer.
		if (intrData[RIFFA_MAX_NUM_CHNLS + chnl].NewTxn == TRUE) {
			// If the user has already called the receive function, start the
			// transaction. If not, set the transaction data and the "ready"
			// bit so that when the user calls the receive function, the
			// transaction can start.
			devExt->Chnl[RIFFA_MAX_NUM_CHNLS + chnl].Length =
				(((UINT64)intrData[RIFFA_MAX_NUM_CHNLS + chnl].Length)<<2);
			devExt->Chnl[RIFFA_MAX_NUM_CHNLS + chnl].Offset =
				(((UINT64)(intrData[RIFFA_MAX_NUM_CHNLS + chnl].OffLast>>1))<<2);
			devExt->Chnl[RIFFA_MAX_NUM_CHNLS + chnl].Last =
				(intrData[RIFFA_MAX_NUM_CHNLS + chnl].OffLast & 0x1);
			KdPrintEx((DPFLTR_IHVDRIVER_ID, DPFLTR_TRACE_LEVEL,
				"riffa: fpga:%s chnl:%d, new recv txn (len:%u off:%u last:%d)\n",
				devExt->Name, chnl, intrData[RIFFA_MAX_NUM_CHNLS + chnl].Length,
				intrData[RIFFA_MAX_NUM_CHNLS + chnl].OffLast>>1,
				intrData[RIFFA_MAX_NUM_CHNLS + chnl].OffLast & 0x1));
			if (InterlockedExchange(&devExt->Chnl[RIFFA_MAX_NUM_CHNLS + chnl].Ready, 2) == 1) {
				// Clear the "ready" bit and start the transaction
				InterlockedExchange(&devExt->Chnl[RIFFA_MAX_NUM_CHNLS + chnl].Ready, 0);
				RiffaStartRecvTransaction(devExt, RIFFA_MAX_NUM_CHNLS + chnl);
			}
		}

		// Send more scatter gather elements.
		if (intrData[chnl].SgRead == TRUE) {
			KdPrintEx((DPFLTR_IHVDRIVER_ID, DPFLTR_TRACE_LEVEL,
				"riffa: fpga:%s chnl:%d, send sg buf read\n", devExt->Name, chnl));
			RiffaProgramScatterGather(devExt, chnl);
		}

		// Finished with downstream transfer.
		if (intrData[chnl].Done == TRUE) {
			// Check if we've requested a done (in the event of a split transaction)
			doneReqd = InterlockedExchange(&devExt->Chnl[chnl].ReqdDone, 0);
			// Indicate this DMA operation has completed. This may result in
			// another transfer if there is still data to be transfered in the request.
			txnComplete = WdfDmaTransactionDmaCompleted(devExt->Chnl[chnl].DmaTransaction, &status);
			if (txnComplete) {
				// Read the actual transfer length
				tnfr = READ_REGISTER_ULONG(devExt->Bar0 + CHNL_REG(chnl, RIFFA_RX_TNFR_LEN_REG));
				KdPrintEx((DPFLTR_IHVDRIVER_ID, DPFLTR_TRACE_LEVEL,
					"riffa: fpga:%s chnl:%d, send txn done\n", devExt->Name, chnl));
				RiffaTransactionComplete(devExt, chnl, tnfr, status);
			}
			else {
				if (doneReqd == 0) {
					// Not complete and not expecting a done signal. Must be an error.
					// End the transaction early. Read the actual transfer length
					tnfr = READ_REGISTER_ULONG(devExt->Bar0 + CHNL_REG(chnl, RIFFA_RX_TNFR_LEN_REG));
					KdPrintEx((DPFLTR_IHVDRIVER_ID, DPFLTR_TRACE_LEVEL,
						"riffa: fpga:%s chnl:%d, send txn done\n", devExt->Name, chnl));
					RiffaTransactionComplete(devExt, chnl, tnfr, STATUS_TRANSACTION_ABORTED);
				}
				else {
					KdPrintEx((DPFLTR_IHVDRIVER_ID, DPFLTR_TRACE_LEVEL,
						"riffa: fpga:%s chnl:%d, send txn split, registers remapped\n", devExt->Name, chnl));
				}
			}
		}
	}
}



/******************************************************************************
 * IOCTL FUNCTIONS
 *****************************************************************************/

/**
 * This event is called when the framework receives IRP_MJ_DEVICE_CONTROL
 * requests from the system.
 *	.
 * Queue - Handle to the framework queue object that is associated with the
 * I/O request.
 *
 * Request - Handle to a framework request object.
 *
 * OutputBufferLength - Length of the request's output buffer,
 * if an output buffer is available.
 *
 * InputBufferLength - Length of the request's input buffer,
 * if an input buffer is available.
 *
 * IoControlCode - the driver-defined or system-defined I/O control code
 * (IOCTL) that is associated with the request.
 */
VOID RiffaEvtIoDeviceControl(IN WDFQUEUE Queue, IN WDFREQUEST Request,
	IN size_t OutputBufferLength, IN size_t InputBufferLength,
	IN ULONG IoControlCode) {
    PDEVICE_EXTENSION devExt;

    devExt = RiffaGetDeviceContext(WdfIoQueueGetDevice(Queue));

    // Determine which I/O control code was specified.
    switch (IoControlCode) {

    case IOCTL_RIFFA_SEND: // (METHOD_OUT_DIRECT)
    	RiffaIoctlSend(devExt, Request, OutputBufferLength, InputBufferLength);
		break;

    case IOCTL_RIFFA_RECV: // (METHOD_OUT_DIRECT)
    	RiffaIoctlRecv(devExt, Request, OutputBufferLength, InputBufferLength);
		break;

    case IOCTL_RIFFA_LIST: // (METHOD_OUT_DIRECT)
    	RiffaIoctlList(devExt, Request, OutputBufferLength, InputBufferLength);
		break;

    case IOCTL_RIFFA_RESET: // (METHOD_OUT_DIRECT)
    	RiffaIoctlReset(devExt, Request);
		break;

    default:
        // The specified I/O control code is unrecognized by this driver.
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: fpga:%s, invalid ioctl request type\n", devExt->Name);
        WdfRequestCompleteWithInformation(Request, STATUS_INVALID_DEVICE_REQUEST, 0);
        break;
    }
}


/**
 * Handles IRP_MJ_DEVICE_CONTROL requests for IOCTL_RIFFA_SEND.
 *	.
 * DevExt - Handle to the device extension object.
 *
 * Request - Handle to a framework request object.
 *
 * OutputBufferLength - Length of the request's output buffer,
 * if an output buffer is available.
 *
 * InputBufferLength - Length of the request's input buffer,
 * if an input buffer is available.
 */
VOID RiffaIoctlSend(IN PDEVICE_EXTENSION DevExt, IN WDFREQUEST Request,
	IN size_t OutputBufferLength, IN size_t InputBufferLength) {
    NTSTATUS status = STATUS_SUCCESS;
    PREQUEST_EXTENSION reqExt;
    PRIFFA_FPGA_CHNL_IO io;
	UINT64 length;
	PCHAR buf = NULL;
	size_t bufSize;

	// Input should be non-zero
	if(!InputBufferLength) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: fpga:%s, ioctl send zero length input buffer\n", DevExt->Name);
		WdfRequestCompleteWithInformation(Request, STATUS_INVALID_PARAMETER, 0);
		return;
	}

	// Get the input buffer
	status = WdfRequestRetrieveInputBuffer(Request, 0, &buf, &bufSize);
	if (!NT_SUCCESS(status)) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: fpga:%s, ioctl send WdfRequestRetrieveInputBuffer failed\n",
			DevExt->Name);
		WdfRequestCompleteWithInformation(Request, status, 0);
		return;
	}
	if (bufSize < sizeof(RIFFA_FPGA_CHNL_IO)) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: fpga:%s, ioctl send input buffer too small to contain RIFFA_FPGA_CHNL_IO\n",
			DevExt->Name);
		WdfRequestCompleteWithInformation(Request, STATUS_INVALID_PARAMETER, 0);
		return;
	}
	io = (PRIFFA_FPGA_CHNL_IO)buf;

	// Validate the length, last, chnl
	length = (io->Length < (OutputBufferLength>>2) ? io->Length : (OutputBufferLength>>2));
	if (io->Last > 1) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: fpga:%s, ioctl send invalid last: %d\n", DevExt->Name, io->Last);
		WdfRequestCompleteWithInformation(Request, STATUS_INVALID_PARAMETER, 0);
		return;
	}
	if (io->Chnl >= DevExt->NumChnls) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: fpga:%s, ioctl send invalid channel: %d, device only has %d channel(s)\n",
			DevExt->Name, io->Chnl, DevExt->NumChnls);
		WdfRequestCompleteWithInformation(Request, STATUS_INVALID_PARAMETER, 0);
		return;
	}

	// Check that this isn't an already running transaction
	if (InterlockedExchange(&DevExt->Chnl[io->Chnl].InUse, 1) == 1) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: fpga:%s, ioctl send already in use on channel: %d\n",
			DevExt->Name, io->Chnl);
		WdfRequestCompleteWithInformation(Request, STATUS_INVALID_PARAMETER, 0);
		return;
	}

	// Set the channel number in the request context for RiffaEvtRequestCancel.
    reqExt = RiffaGetRequestContext(Request);
    reqExt->Chnl = io->Chnl;

	// Start a send transaction.
	if (length) {
		// Start a DMA transaction for sending.
		DevExt->Chnl[io->Chnl].Timeout = io->Timeout;
		DevExt->Chnl[io->Chnl].Length = (length<<2);
		DevExt->Chnl[io->Chnl].SpillAfter = (length<<2);
		DevExt->Chnl[io->Chnl].Offset = (io->Offset<<2);
		DevExt->Chnl[io->Chnl].Last = io->Last;
		DevExt->Chnl[io->Chnl].Provided = 0;
		DevExt->Chnl[io->Chnl].ProvidedPrev = 0;
		DevExt->Chnl[io->Chnl].Confirmed = 0;
		DevExt->Chnl[io->Chnl].ConfirmedPrev = 0;
		DevExt->Chnl[io->Chnl].ReqdDone = 0;

		// Get the user space memory.
		status = WdfRequestRetrieveOutputWdmMdl(Request, &DevExt->Chnl[io->Chnl].Mdl);
		if (!NT_SUCCESS(status)) {
			DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
				"riffa: fpga:%s, WdfRequestRetrieveOutputWdmMdl failed\n", DevExt->Name);
			WdfRequestCompleteWithInformation(Request, status, 0);
			return;
		}

		// Put the WDFREQUEST into our pending queue, complete later
        status = WdfRequestForwardToIoQueue(Request, DevExt->Chnl[io->Chnl].PendingQueue);
		if (!NT_SUCCESS(status)) {
			DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
				"riffa: fpga:%s, ioctl send WdfRequestForwardToIoQueue failed\n",
				DevExt->Name);
			WdfRequestCompleteWithInformation(Request, status, 0);
			return;
		}

		// Actually start the transaction
		status = RiffaStartDmaTransaction(DevExt, io->Chnl, (length<<2),
			0, WdfDmaDirectionWriteToDevice);
		if (!NT_SUCCESS(status)) {
			WdfRequestCompleteWithInformation(Request, status, 0);
			return;
		}
	}
	else if (io->Last) {
		// Program the device for zero length send (device RX) and complete
		RiffaProgramSend(DevExt, io->Chnl, (UINT32)length, io->Offset, io->Last);
		WdfRequestCompleteWithInformation(Request, STATUS_SUCCESS, 0);
		return;
	}
	else {
		// Invalid request, results in no send
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: fpga:%s, ioctl send invalid, no length or last\n", DevExt->Name);
		WdfRequestCompleteWithInformation(Request, STATUS_INVALID_PARAMETER, 0);
		return;
	}
}



/**
 * Handles IRP_MJ_DEVICE_CONTROL requests for IOCTL_RIFFA_RECV.
 *	.
 * DevExt - Handle to the device extension object.
 *
 * Request - Handle to a framework request object.
 *
 * OutputBufferLength - Length of the request's output buffer,
 * if an output buffer is available.
 *
 * InputBufferLength - Length of the request's input buffer,
 * if an input buffer is available.
 */
VOID RiffaIoctlRecv(IN PDEVICE_EXTENSION DevExt, IN WDFREQUEST Request,
	IN size_t OutputBufferLength, IN size_t InputBufferLength) {
    NTSTATUS status = STATUS_SUCCESS;
	PREQUEST_EXTENSION reqExt;
    PRIFFA_FPGA_CHNL_IO io;
	UINT64 length;
	PCHAR buf = NULL;
	size_t bufSize;

	// Input should be non-zero
	if(!InputBufferLength) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: fpga:%s, ioctl recv zero length input buffer\n", DevExt->Name);
		WdfRequestCompleteWithInformation(Request, STATUS_INVALID_PARAMETER, 0);
		return;
	}

	// Get the input buffer
	status = WdfRequestRetrieveInputBuffer(Request, 0, &buf, &bufSize);
	if (!NT_SUCCESS(status)) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: fpga:%s, ioctl recv WdfRequestRetrieveInputBuffer failed\n",
			DevExt->Name);
		WdfRequestCompleteWithInformation(Request, status, 0);
		return;
	}
	if (bufSize < sizeof(RIFFA_FPGA_CHNL_IO)) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: fpga:%s, ioctl recv input buffer too small to contain RIFFA_FPGA_CHNL_IO\n",
			DevExt->Name);
		WdfRequestCompleteWithInformation(Request, STATUS_INVALID_PARAMETER, 0);
		return;
	}
	io = (PRIFFA_FPGA_CHNL_IO)buf;

	// Validate the length, chnl
	length = (io->Length < (OutputBufferLength>>2) ? io->Length : (OutputBufferLength>>2));
	if (io->Chnl >= DevExt->NumChnls) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: fpga:%s, ioctl recv invalid channel: %d, device only has %d channel(s)\n",
			DevExt->Name, io->Chnl, DevExt->NumChnls);
		WdfRequestCompleteWithInformation(Request, STATUS_INVALID_PARAMETER, 0);
		return;
	}

	// Check that this isn't an already running transaction
	if (InterlockedExchange(&DevExt->Chnl[RIFFA_MAX_NUM_CHNLS + io->Chnl].InUse, 1) == 1) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: fpga:%s, ioctl recv already in use on channel: %d\n",
			DevExt->Name, io->Chnl);
		WdfRequestCompleteWithInformation(Request, STATUS_INVALID_PARAMETER, 0);
		return;
	}

	// Set the channel number in the request context for RiffaEvtRequestCancel.
    reqExt = RiffaGetRequestContext(Request);
    reqExt->Chnl = RIFFA_MAX_NUM_CHNLS + io->Chnl;

	// Start a receive transaction. If an interrupt with the transaction
	// info has already been received, start the transaction. If not, set
	// this transaction request and the "ready" bit so that when the
	// interrupt is received the transaction can start.
	DevExt->Chnl[RIFFA_MAX_NUM_CHNLS + io->Chnl].Timeout = io->Timeout;
	DevExt->Chnl[RIFFA_MAX_NUM_CHNLS + io->Chnl].Capacity = (length<<2);
	DevExt->Chnl[RIFFA_MAX_NUM_CHNLS + io->Chnl].Provided = 0;
	DevExt->Chnl[RIFFA_MAX_NUM_CHNLS + io->Chnl].ProvidedPrev = 0;
	DevExt->Chnl[RIFFA_MAX_NUM_CHNLS + io->Chnl].Confirmed = 0;
	DevExt->Chnl[RIFFA_MAX_NUM_CHNLS + io->Chnl].ConfirmedPrev = 0;
	DevExt->Chnl[RIFFA_MAX_NUM_CHNLS + io->Chnl].ReqdDone = 0;

	// Get the user space memory.
	status = WdfRequestRetrieveOutputWdmMdl(Request,
		&DevExt->Chnl[RIFFA_MAX_NUM_CHNLS + io->Chnl].Mdl);
	if (!NT_SUCCESS(status)) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: fpga:%s, WdfRequestRetrieveOutputWdmMdl failed\n", DevExt->Name);
		WdfRequestCompleteWithInformation(Request, status, 0);
		return;
	}

	// Put the WDFREQUEST into our pending queue, complete later
    status = WdfRequestForwardToIoQueue(Request,
    	DevExt->Chnl[RIFFA_MAX_NUM_CHNLS + io->Chnl].PendingQueue);
	if (!NT_SUCCESS(status)) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: fpga:%s, ioctl send WdfRequestForwardToIoQueue failed\n", DevExt->Name);
		WdfRequestCompleteWithInformation(Request, status, 0);
		return;
	}

	// Start the timer (if necessary)
	if (DevExt->Chnl[RIFFA_MAX_NUM_CHNLS + io->Chnl].Timeout > 0)
		WdfTimerStart(DevExt->Chnl[RIFFA_MAX_NUM_CHNLS + io->Chnl].Timer,
			WDF_REL_TIMEOUT_IN_MS(DevExt->Chnl[RIFFA_MAX_NUM_CHNLS + io->Chnl].Timeout));

	// See if the FPGA set the Ready bit
	if (InterlockedExchange(&DevExt->Chnl[RIFFA_MAX_NUM_CHNLS + io->Chnl].Ready, 1) == 2) {
		// Clear the "ready" bit and start the transaction
		InterlockedExchange(&DevExt->Chnl[RIFFA_MAX_NUM_CHNLS + io->Chnl].Ready, 0);
		RiffaStartRecvTransaction(DevExt, RIFFA_MAX_NUM_CHNLS + io->Chnl);
		return;
	}
}



/**
 * Handles IRP_MJ_DEVICE_CONTROL requests for IOCTL_RIFFA_LIST.
 *	.
 * DevExt - Handle to the device extension object.
 *
 * Request - Handle to a framework request object.
 *
 * OutputBufferLength - Length of the request's output buffer,
 * if an output buffer is available.
 *
 * InputBufferLength - Length of the request's input buffer,
 * if an input buffer is available.
 */
VOID RiffaIoctlList(IN PDEVICE_EXTENSION DevExt, IN WDFREQUEST Request,
	IN size_t OutputBufferLength, IN size_t InputBufferLength) {
    NTSTATUS status = STATUS_SUCCESS;
    PRIFFA_FPGA_INFO info;
	PCHAR buf = NULL;
	size_t bufSize;
	UINT32 i;

	// Input should be non-zero
	if(!InputBufferLength) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: fpga:%s, ioctl list zero length input buffer\n", DevExt->Name);
		WdfRequestCompleteWithInformation(Request, STATUS_INVALID_PARAMETER, 0);
		return;
	}

	// Get the input buffer
	status = WdfRequestRetrieveInputBuffer(Request, 0, &buf, &bufSize);
	if (!NT_SUCCESS(status)) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: fpga:%s, ioctl list WdfRequestRetrieveInputBuffer failed\n",
			DevExt->Name);
		WdfRequestCompleteWithInformation(Request, status, 0);
		return;
	}
	if (bufSize < sizeof(UINT32)) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: fpga:%s, ioctl list input buffer too small to contain fpga id\n",
			DevExt->Name);
		WdfRequestCompleteWithInformation(Request, STATUS_INVALID_PARAMETER, 0);
		return;
	}
	i = *((UINT32 *)buf);

	// Validate the index value
	if (i >= RIFFA_MAX_NUM_FPGAS) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: fpga:%s, ioctl list fpga id invalid\n", DevExt->Name);
		WdfRequestCompleteWithInformation(Request, STATUS_INVALID_PARAMETER, 0);
		return;
	}

	// Output should be non-zero
	if(!OutputBufferLength) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: fpga:%s, ioctl list zero length output buffer\n", DevExt->Name);
		WdfRequestCompleteWithInformation(Request, STATUS_INVALID_PARAMETER, 0);
		return;
	}

	// Get the output buffer
	status = WdfRequestRetrieveOutputBuffer(Request, 0, &buf, &bufSize);
	if (!NT_SUCCESS(status)) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: fpga:%s, ioctl list WdfRequestRetrieveOutputBuffer failed\n",
			DevExt->Name);
		WdfRequestCompleteWithInformation(Request, status, 0);
		return;
	}
	if (bufSize < sizeof(RIFFA_FPGA_INFO)) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: fpga:%s, ioctl list input buffer too small to contain RIFFA_FPGA_INFO\n",
			DevExt->Name);
		WdfRequestCompleteWithInformation(Request, STATUS_INVALID_PARAMETER, 0);
		return;
	}
	info = (PRIFFA_FPGA_INFO)buf;

	// Fill in the information.
	info->id[i] = i;
	info->num_chnls[i] = DevExt->NumChnls;
	strcpy_s(info->name[i], 16, DevExt->Name);
	info->vendor_id[i] = DevExt->VendorId;
	info->device_id[i] = DevExt->DeviceId;
	WdfRequestCompleteWithInformation(Request, status, 0);
}



/**
 * Handles IRP_MJ_DEVICE_CONTROL requests for IOCTL_RIFFA_RESET.
 *	.
 * DevExt - Handle to the device extension object.
 *
 * Request - Handle to a framework request object.
 *
 * OutputBufferLength - Length of the request's output buffer,
 * if an output buffer is available.
 *
 * InputBufferLength - Length of the request's input buffer,
 * if an input buffer is available.
 */
VOID RiffaIoctlReset(IN PDEVICE_EXTENSION DevExt, IN WDFREQUEST Request) {
	UINT32 i;

	// Reset the device by reading the info/status register
	READ_REGISTER_ULONG(DevExt->Bar0 + RIFFA_INFO_REG);

	// Reset all the channels
	for (i = 0; i < DevExt->NumChnls; i++) {
		RiffaCompleteRequest(DevExt, i, STATUS_CANCELLED, FALSE);
		RiffaCompleteRequest(DevExt, RIFFA_MAX_NUM_CHNLS + i, STATUS_CANCELLED, FALSE);
	}

	// Reset the interrupt data
	memset(DevExt->IntrData, 0, 2 * RIFFA_MAX_NUM_CHNLS * sizeof(INTR_CHNL_DIR_DATA));

	// Finish this request
	WdfRequestCompleteWithInformation(Request, STATUS_SUCCESS, 0);
}



/******************************************************************************
 * DMA ENTRY, EXIT, TIMER FUNCTIONS
 *****************************************************************************/

/**
 * Called when the WDFREQUEST object for the specified channel should be
 * completed with the specified status.
 *
 * DevExt - Pointer to the Device Extension
 *
 * Chnl - Channel number on which the DMA is taking place
 *
 * Status - NTSTATUS to set for completion
 *
 * TimedOut - True if this is a timeout completion, false otherwise
 */
VOID RiffaCompleteRequest(IN PDEVICE_EXTENSION DevExt, IN UINT32 Chnl,
	IN NTSTATUS Status, BOOLEAN TimedOut) {
	WDFREQUEST request;
	UINT64 total;

	NTSTATUS status;
	UINT32 i;

	// Stop the timer
	WdfTimerStop(DevExt->Chnl[Chnl].Timer, FALSE);

	// Releas the transaction
	WdfDmaTransactionRelease(DevExt->Chnl[Chnl].DmaTransaction);

	// Get the request (if not already cancelled).
	WdfIoQueueRetrieveNextRequest(DevExt->Chnl[Chnl].PendingQueue, &request);

	// Quick calculation of total words
	total = (DevExt->Chnl[Chnl].ConfirmedPrev + DevExt->Chnl[Chnl].Confirmed)>>2;

	// Clear the status bits
	DevExt->Chnl[Chnl].InUse = 0;
	DevExt->Chnl[Chnl].Ready = 0;
	DevExt->Chnl[Chnl].ReqdDone = 0;
	DevExt->Chnl[Chnl].Capacity = 0;
	DevExt->Chnl[Chnl].Provided = 0;
	DevExt->Chnl[Chnl].Length = 0;
	DevExt->Chnl[Chnl].SpillAfter = 0;
	DevExt->Chnl[Chnl].Timeout = 0;

	if (request != NULL) {
		if (TimedOut) {
			if (Chnl < RIFFA_MAX_NUM_CHNLS) {
				DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
					"riffa: fpga:%s chnl:%d, send timed out\n", DevExt->Name, Chnl);
			}
			else {
				DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
					"riffa: fpga:%s chnl:%d, recv timed out\n", DevExt->Name,
					Chnl - RIFFA_MAX_NUM_CHNLS);
			}
		}
		WdfRequestCompleteWithInformation(request, Status, (ULONG_PTR)total);
	}
}

/**
 * Called when the WDFTIMER expires. Used to handle timeouts from IOCTL calls.
 *
 * Timer - WDFTIMER that expired
 */
VOID RiffaEvtTimerFunc(IN WDFTIMER Timer) {
	PDEVICE_EXTENSION devExt;
	PTIMER_EXTENSION timerExt;
	WDFREQUEST request;
	NTSTATUS status;
	BOOLEAN canCancel;

    timerExt = RiffaGetTimerContext(Timer);
    devExt = RiffaGetDeviceContext(WdfIoQueueGetDevice(WdfTimerGetParentObject(Timer)));

	// Cancel the request
	RiffaCompleteRequest(devExt, timerExt->Chnl, STATUS_CANCELLED, TRUE);
}


/******************************************************************************
 * DMA CALLBACKS
 *****************************************************************************/

/**
 * Called after the user has called into the driver to receive data, from the
 * interrupt DPC when it detects a data receive event. Both conditions must be
 * met before this function will be called. The function initiates a receive
 * DMA operation then provides scatter gather information if necessary.
 *
 * DevExt - Pointer to the Device Extension
 *
 * Chnl - Channel number on which the DMA is taking place
 */
VOID RiffaStartRecvTransaction(IN PDEVICE_EXTENSION DevExt, IN UINT32 Chnl) {
	NTSTATUS status;
	UINT64 length;

	// Stop the timer (if set)
	WdfTimerStop(DevExt->Chnl[Chnl].Timer, FALSE);

	// Start a recv transaction.
	if (DevExt->Chnl[Chnl].Length) {
		// Calculate room in the user space buffer and what needs to be spilled
		if (DevExt->Chnl[Chnl].Capacity > DevExt->Chnl[Chnl].Offset) {
			// Some (possibly all) of the data can fit in the user buffer.
			DevExt->Chnl[Chnl].SpillAfter =
				DevExt->Chnl[Chnl].Capacity - DevExt->Chnl[Chnl].Offset;
			length = (DevExt->Chnl[Chnl].Capacity < DevExt->Chnl[Chnl].Length ?
				DevExt->Chnl[Chnl].Capacity : DevExt->Chnl[Chnl].Length);
			status = RiffaStartDmaTransaction(DevExt, Chnl,
				length + DevExt->Chnl[Chnl].Offset, DevExt->Chnl[Chnl].Offset,
				WdfDmaDirectionReadFromDevice);
			if (!NT_SUCCESS(status)) {
				RiffaCompleteRequest(DevExt, Chnl, status, FALSE);
			}
		}
		else {
			// No room in user buffer, spill everything
			DevExt->Chnl[Chnl].SpillAfter = 0;
			RiffaProgramScatterGather(DevExt, Chnl);
		}
	}
	else if (DevExt->Chnl[Chnl].Last) {
		// Recognize zero length receive and complete
		RiffaCompleteRequest(DevExt, Chnl, STATUS_SUCCESS, FALSE);
	}
	else {
		// Invalid request, should never happen
		RiffaCompleteRequest(DevExt, Chnl, STATUS_CANCELLED, FALSE);
	}
}


/**
 * Starts a DMA transaction using the user pages specified in the IOCTL output
 * buffer as the source/receptical. This will result in RiffaEvtProgramDma
 * being called with the scatter gather list. If the amount of data to map is
 * too large, Windows will break up the transaction into multiple transfers.
 * Only a portion of the output buffer will be mapped each transfer (each call
 * to RiffaEvtProgramDma). Returns the status after starting the DMA transaction.
 *
 * Request - Pointer to the WDFREQUEST that was passed to IOCTl
 *
 * DevExt - Pointer to the Device Extension
 *
 * Chnl - Channel number on which the DMA transaction should take place
 *
 * Length - Length (in bytes) of the DMA
 *
 * Offset - Offset (in bytes) from the start of the IOCTL output buffer where
 * data should be read/written
 *
 * DmaDirection - Direction of the DMA
 *
 * Returns status after starting the DMA transaction.
 */
NTSTATUS RiffaStartDmaTransaction(IN PDEVICE_EXTENSION DevExt, IN UINT32 Chnl,
	IN UINT64 Length, IN UINT64 Offset, IN WDF_DMA_DIRECTION DmaDirection) {
	NTSTATUS status = STATUS_SUCCESS;
    PVOID vaddr;
    UINT64 length;

	DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
		"riffa: fpga:%s chnl:%d, starting txn (len:%lld, off:%lld, isSend?:%d)\n",
		DevExt->Name, (Chnl < RIFFA_MAX_NUM_CHNLS ? Chnl : Chnl - RIFFA_MAX_NUM_CHNLS),
		Length>>2, Offset, DmaDirection == WdfDmaDirectionWriteToDevice);

	// Move the virtual address forward to the offset
	vaddr = MmGetMdlVirtualAddress(DevExt->Chnl[Chnl].Mdl);
	vaddr = ((UINT32 *)vaddr) + Offset;

	// Reduce the length by the offset amount
	length = MmGetMdlByteCount(DevExt->Chnl[Chnl].Mdl);
	length = (Length < length ? Length : length);
	length = length - Offset;

	// Reuse the DMA Transaction
	status = WdfDmaTransactionInitialize(DevExt->Chnl[Chnl].DmaTransaction,
		RiffaEvtProgramDma, DmaDirection, DevExt->Chnl[Chnl].Mdl, vaddr, (size_t)length);
	if(!NT_SUCCESS(status)) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: fpga:%s chnl:%d, WdfDmaTransactionInitialize failed\n",
			DevExt->Name, (Chnl < RIFFA_MAX_NUM_CHNLS ? Chnl : Chnl - RIFFA_MAX_NUM_CHNLS));
		return status;
	}

	// Execute the transaction
	status = WdfDmaTransactionExecute(DevExt->Chnl[Chnl].DmaTransaction,
		(WDFCONTEXT)Chnl);
	if (!NT_SUCCESS(status)) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: fpga:%s chnl:%d, WdfDmaTransactionExecute failed\n",
			DevExt->Name, (Chnl < RIFFA_MAX_NUM_CHNLS ? Chnl : Chnl - RIFFA_MAX_NUM_CHNLS));
		WdfDmaTransactionRelease(DevExt->Chnl[Chnl].DmaTransaction);
		return status;
	}

	return status;
}



/**
 * Programs the device with scatter gather data. Will program the device with
 * spill buffer data (as scatter gather buffers) if the DMA length will overrun
 * the receptical user buffer.
 *
 * DevExt - Pointer to the Device Extension
 *
 * Chnl - Channel number on which the DMA transaction takes place
 *
 * IsSend - TRUE if the direction is a "send", FALSE otherwise
 */
VOID RiffaProgramScatterGather(IN PDEVICE_EXTENSION DevExt, IN UINT32 Chnl) {
	PSCATTER_GATHER_LIST sgList;
	PULONG bufBase;
	PHYSICAL_ADDRESS bufAddr;
    UINT32 pos;
    UINT32 i;
    UINT64 provided;
    UINT64 length;
    UINT64 spillAfter;

	// Stop the timer (if set)
	WdfTimerStop(DevExt->Chnl[Chnl].Timer, FALSE);

	// OK, then get the variables.
	sgList = DevExt->Chnl[Chnl].SgList;
	bufBase = DevExt->Chnl[Chnl].CommonBufferBase;
	bufAddr = DevExt->Chnl[Chnl].CommonBufferBaseLA;
	pos = DevExt->Chnl[Chnl].SgPos;
	provided = DevExt->Chnl[Chnl].Provided;
	length = DevExt->Chnl[Chnl].Length;
	spillAfter = DevExt->Chnl[Chnl].SpillAfter;

    // Translate the System's SCATTER_GATHER_LIST elements
    // into the device's scatter gather elements.
	for (i = 0; i < DevExt->MaxNumScatterGatherElems && provided < spillAfter; i++, pos++) {
		if (pos >= sgList->NumberOfElements)
			break;
		WRITE_REGISTER_ULONG(bufBase + (i*4) + 0, sgList->Elements[pos].Address.LowPart);
		WRITE_REGISTER_ULONG(bufBase + (i*4) + 1, sgList->Elements[pos].Address.HighPart);
		WRITE_REGISTER_ULONG(bufBase + (i*4) + 2, (sgList->Elements[pos].Length>>2)); // Words!
		provided += sgList->Elements[pos].Length;
	}

	// Handle any spillage
	if (provided >= spillAfter) {
		for (; i < DevExt->MaxNumScatterGatherElems && provided < length; i++) {
			WRITE_REGISTER_ULONG(bufBase + (i*4) + 0, DevExt->SpillBufferBaseLA.LowPart);
			WRITE_REGISTER_ULONG(bufBase + (i*4) + 1, DevExt->SpillBufferBaseLA.HighPart);
			WRITE_REGISTER_ULONG(bufBase + (i*4) + 2, (RIFFA_SPILL_BUF_SIZE>>2)); // Words!
			provided += RIFFA_SPILL_BUF_SIZE;
		}
	}

    // Update the data structures
    DevExt->Chnl[Chnl].Confirmed += DevExt->Chnl[Chnl].ProvidedPrev;
    DevExt->Chnl[Chnl].ProvidedPrev = provided - DevExt->Chnl[Chnl].Provided;
	DevExt->Chnl[Chnl].Provided = provided;
	DevExt->Chnl[Chnl].SgPos = pos;

	if (i > 0) {
		// Let the device know about the new scatter gather data.
		if (Chnl < RIFFA_MAX_NUM_CHNLS) {
			KdPrintEx((DPFLTR_IHVDRIVER_ID, DPFLTR_TRACE_LEVEL,
				"riffa: fpga:%s chnl:%d, sg buf for send txn: %d elements\n",
				DevExt->Name, (Chnl < RIFFA_MAX_NUM_CHNLS ? Chnl : Chnl - RIFFA_MAX_NUM_CHNLS), i));
			WRITE_REGISTER_ULONG(DevExt->Bar0 + CHNL_REG(Chnl, RIFFA_RX_SG_ADDR_LO_REG), bufAddr.LowPart);
			WRITE_REGISTER_ULONG(DevExt->Bar0 + CHNL_REG(Chnl, RIFFA_RX_SG_ADDR_HI_REG), bufAddr.HighPart);
			WRITE_REGISTER_ULONG(DevExt->Bar0 + CHNL_REG(Chnl, RIFFA_RX_SG_LEN_REG), (i*4*4)>>2); // Words!
		}
		else {
			KdPrintEx((DPFLTR_IHVDRIVER_ID, DPFLTR_TRACE_LEVEL,
				"riffa: fpga:%s chnl:%d, sg buf for recv txn: %d elements\n",
				DevExt->Name, (Chnl < RIFFA_MAX_NUM_CHNLS ? Chnl : Chnl - RIFFA_MAX_NUM_CHNLS), i));
			WRITE_REGISTER_ULONG(DevExt->Bar0 + CHNL_REG(Chnl - RIFFA_MAX_NUM_CHNLS, RIFFA_TX_SG_ADDR_LO_REG), bufAddr.LowPart);
			WRITE_REGISTER_ULONG(DevExt->Bar0 + CHNL_REG(Chnl - RIFFA_MAX_NUM_CHNLS, RIFFA_TX_SG_ADDR_HI_REG), bufAddr.HighPart);
			WRITE_REGISTER_ULONG(DevExt->Bar0 + CHNL_REG(Chnl - RIFFA_MAX_NUM_CHNLS, RIFFA_TX_SG_LEN_REG), (i*4*4)>>2); // Words!
		}
	}
	else if (provided < length) {
		// Let the device know that no more scatter gather data will be provided
		// until a "transfer done" signal is received. This is initiated differently
		// for a send vs. a receive transaction.
		InterlockedExchange(&DevExt->Chnl[Chnl].ReqdDone, 1);
		if (Chnl < RIFFA_MAX_NUM_CHNLS) {
			// Write the length to signal request for "done" signal after all the
			// scatter gather regions have been used.
		    WRITE_REGISTER_ULONG(DevExt->Bar0 + CHNL_REG(Chnl, RIFFA_RX_LEN_REG),
		    	(ULONG)(provided>>2));
		}
		else {
			// Read the length to signal request for "done" signal after all the
			// scatter gather regions have been used.
			READ_REGISTER_ULONG(DevExt->Bar0 + CHNL_REG(Chnl - RIFFA_MAX_NUM_CHNLS, RIFFA_TX_LEN_REG));
		}
		if (Chnl < RIFFA_MAX_NUM_CHNLS) {
			KdPrintEx((DPFLTR_IHVDRIVER_ID, DPFLTR_TRACE_LEVEL,
				"riffa: fpga:%s chnl:%d, splitting send txn\n", DevExt->Name, Chnl));
		}
		else {
			KdPrintEx((DPFLTR_IHVDRIVER_ID, DPFLTR_TRACE_LEVEL,
				"riffa: fpga:%s chnl:%d, splitting recv txn\n", DevExt->Name, Chnl - RIFFA_MAX_NUM_CHNLS));
		}
	}

	// Start the timer (if necessary)
	if (DevExt->Chnl[Chnl].Timeout > 0)
		WdfTimerStart(DevExt->Chnl[Chnl].Timer, WDF_REL_TIMEOUT_IN_MS(DevExt->Chnl[Chnl].Timeout));
}



/**
 * Called after the last DMA transfer for the DMA transaction is completed.
 * Sets the transferred bytes and completes the IOCTL call.
 *
 * DevExt - Pointer to the Device Extension
 *
 * Chnl - Channel number on which the DMA transaction takes place
 *
 * Transferred - Words transferred according to the hardware
 *
 * Status - Status after the last transfer has completed.
 */
VOID RiffaTransactionComplete(IN PDEVICE_EXTENSION DevExt, IN UINT32 Chnl,
	IN UINT32 Transferred, IN NTSTATUS Status) {
	WDFREQUEST request;
	NTSTATUS status;

	// Stop the timer (if set)
	WdfTimerStop(DevExt->Chnl[Chnl].Timer, FALSE);

	// Release the DMA Transaction (ok to call this multiple times)
	WdfDmaTransactionRelease(DevExt->Chnl[Chnl].DmaTransaction);

	if (Chnl < RIFFA_MAX_NUM_CHNLS) {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: fpga:%s chnl:%d, words transferred: %lu\n", DevExt->Name, Chnl, Transferred);

		// Update the total confirmed data
		DevExt->Chnl[Chnl].Confirmed = (((UINT64)Transferred)<<2);

		// Complete the send request
		RiffaCompleteRequest(DevExt, Chnl, Status, FALSE);
	}
	else {
		DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL,
			"riffa: fpga:%s chnl:%d, words transferred: %lu\n", DevExt->Name,
			Chnl - RIFFA_MAX_NUM_CHNLS, Transferred);

		// Complete or repeat
		if (DevExt->Chnl[Chnl].Last || !NT_SUCCESS(Status)) {
			// Update the total confirmed data
			DevExt->Chnl[Chnl].Confirmed = (((UINT64)Transferred)<<2);

			// Complete the receive request
			RiffaCompleteRequest(DevExt, Chnl, Status, FALSE);
		}
		else {
			// Not the "last" transaction. Save the transferred amount.
			DevExt->Chnl[Chnl].ConfirmedPrev += (((UINT64)Transferred)<<2);
			DevExt->Chnl[Chnl].Confirmed = 0;
			DevExt->Chnl[Chnl].ProvidedPrev = 0;
			DevExt->Chnl[Chnl].Provided = 0;

			// Stay in the kernel and start another receive DMA transaction.
			// If an interrupt with transaction info has already been received,
			// start the transaction. If not, set the "ready" bit.
			if (InterlockedExchange(&DevExt->Chnl[Chnl].Ready, 1) == 2) {
				// Clear the "ready" bit and start the transaction
				InterlockedExchange(&DevExt->Chnl[Chnl].Ready, 0);
				RiffaStartRecvTransaction(DevExt, Chnl);
			}
		}
	}
}



/**
 * Programs the device for DMA send transactions. Note, that there is no way to
 * program the device for receive transactions. Receive transactions can only
 * be initiated from the device.
 *
 * DevExt - Pointer to the Device Extension
 *
 * Chnl - Channel number on which the DMA transaction takes place
 *
 * Length - Length (in words) of the DMA
 *
 * Offset - Offset (in words) to send to the device
 *
 * Last - 1 if this is the last transaction for the device, 0 otherwise
 */
VOID RiffaProgramSend(IN PDEVICE_EXTENSION DevExt, IN UINT32 Chnl, IN UINT32 Length,
	IN UINT32 Offset, IN UINT32 Last) {
    UINT32 offlast;
    UINT32 chnl;

    // Let the device know about the new transfer.
    offlast = ((Offset<<1) | (Last & 0x1));
    chnl = (Chnl >= RIFFA_MAX_NUM_CHNLS ? Chnl - RIFFA_MAX_NUM_CHNLS : Chnl);
    WRITE_REGISTER_ULONG(DevExt->Bar0 + CHNL_REG(chnl, RIFFA_RX_OFFLAST_REG), offlast);
    WRITE_REGISTER_ULONG(DevExt->Bar0 + CHNL_REG(chnl, RIFFA_RX_LEN_REG), Length);
}



/**
 * The framework calls a driver's EvtProgramDma event callback function
 * when the driver calls WdfDmaTransactionExecute and the system has
 * enough map registers to do the transfer. The callback function must
 * program the hardware to start the transfer. A single transaction
 * initiated by calling WdfDmaTransactionExecute may result in multiple
 * calls to this function if the buffer is too large and there aren't
 * enough map registers to do the whole transfer.
 *
 * Transaction - DMA transactions initialized and started in RiffaStartDmaTransaction
 *
 * Device - A handle to the WDFDEVICE
 *
 * Context - Value passed as the context parameter to WdfDmaTransactionExecute
 *
 * Direction - DMA direction of the transfer
 *
 * SgList - Scatter gather list of the mapped memory to use for transferring
 *
 * Returns TRUE
 */
BOOLEAN RiffaEvtProgramDma(IN WDFDMATRANSACTION Transaction, IN WDFDEVICE Device,
	IN PVOID Context, IN WDF_DMA_DIRECTION Direction, IN PSCATTER_GATHER_LIST SgList) {
    PDEVICE_EXTENSION devExt;
	UINT32 chnl;

    // Initialize variables and structures
    devExt = RiffaGetDeviceContext(Device);
	chnl = (UINT32)Context;

	// Save the scatter gather list
	devExt->Chnl[chnl].SgList = SgList;
	devExt->Chnl[chnl].SgPos = 0;

	if (Direction == WdfDmaDirectionWriteToDevice &&
		WdfDmaTransactionGetBytesTransferred(Transaction) == 0) {
		// Program the start of a new transfer.
		RiffaProgramSend(devExt, chnl, (UINT32)(devExt->Chnl[chnl].Length>>2),
			(UINT32)(devExt->Chnl[chnl].Offset>>2), devExt->Chnl[chnl].Last);
	}

	// Write the initial scatter gather data and notify the device.
	RiffaProgramScatterGather(devExt, chnl);

    return TRUE;
}

